deltas: Rework format to allow streaming
authorColin Walters <walters@verbum.org>
Thu, 29 Jan 2015 06:03:55 +0000 (01:03 -0500)
committerColin Walters <walters@verbum.org>
Mon, 16 Feb 2015 15:10:35 +0000 (10:10 -0500)
There's still some silliness here, but there is now only one opcode
open-splice-and-close, that writes a single chunk from the payload.
This is really all we need for metadata, and small content objects are
also fine with this.

We get some deduplication between content objects by creating a
dictionary for (uid,gid,mode) tuples and xattrs.

This still keeps the operation/payload code in, so we could do
rollsums in a future update easily.

src/libostree/ostree-repo-static-delta-compilation.c
src/libostree/ostree-repo-static-delta-private.h
src/libostree/ostree-repo-static-delta-processing.c

index 8b5d8387d9bb9ff94b8325d8e8f0b8c0fb6f5593..6dbd3e19ef98e1b658ac8b039190c9d570cccc12 100644 (file)
@@ -38,6 +38,10 @@ typedef struct {
   GPtrArray *objects;
   GString *payload;
   GString *operations;
+  GHashTable *mode_set; /* GVariant(uuu) -> offset */
+  GPtrArray *modes;
+  GHashTable *xattr_set; /* GVariant(ayay) -> offset */
+  GPtrArray *xattrs;
 } OstreeStaticDeltaPartBuilder;
 
 typedef struct {
@@ -57,9 +61,88 @@ ostree_static_delta_part_builder_unref (OstreeStaticDeltaPartBuilder *part_build
     g_string_free (part_builder->payload, TRUE);
   if (part_builder->operations)
     g_string_free (part_builder->operations, TRUE);
+  g_hash_table_unref (part_builder->mode_set);
+  g_ptr_array_unref (part_builder->modes);
+  g_hash_table_unref (part_builder->xattr_set);
+  g_ptr_array_unref (part_builder->xattrs);
   g_free (part_builder);
 }
 
+static guint
+mode_chunk_hash (const void *vp)
+{
+  GVariant *v = (GVariant*)vp;
+  guint uid, gid, mode;
+  g_variant_get (v, "(uuu)", &uid, &gid, &mode);
+  return uid + gid + mode;
+}
+
+static gboolean
+mode_chunk_equals (const void *one, const void *two)
+{
+  GVariant *v1 = (GVariant*)one;
+  GVariant *v2 = (GVariant*)two;
+  guint uid1, gid1, mode1;
+  guint uid2, gid2, mode2;
+
+  g_variant_get (v1, "(uuu)", &uid1, &gid1, &mode1);
+  g_variant_get (v2, "(uuu)", &uid2, &gid2, &mode2);
+
+  return uid1 == uid2 && gid1 == gid2 && mode1 == mode2;
+}
+
+static guint
+bufhash (const void *b, gsize len)
+{
+  const signed char *p, *e;
+  guint32 h = 5381;
+
+  for (p = (signed char *)b, e = (signed char *)b + len; p != e; p++)
+    h = (h << 5) + h + *p;
+
+  return h;
+}
+
+static guint
+xattr_chunk_hash (const void *vp)
+{
+  GVariant *v = (GVariant*)vp;
+  gsize n = g_variant_n_children (v);
+  guint i;
+  guint32 h = 5381;
+
+  for (i = 0; i < n; i++)
+    {
+      const guint8* name;
+      const guint8* value_data;
+      GVariant *value = NULL;
+      gsize value_len;
+
+      g_variant_get_child (v, i, "(^&ay@ay)",
+                           &name, &value);
+      value_data = g_variant_get_fixed_array (value, &value_len, 1);
+      
+      h += g_str_hash (name);
+      h += bufhash (value_data, value_len);
+    }
+      
+  return h;
+}
+
+static gboolean
+xattr_chunk_equals (const void *one, const void *two)
+{
+  GVariant *v1 = (GVariant*)one;
+  GVariant *v2 = (GVariant*)two;
+  gsize l1 = g_variant_get_size (v1);
+  gsize l2 = g_variant_get_size (v2);
+
+  if (l1 != l2)
+    return FALSE;
+
+  return memcmp (g_variant_get_data (v1), g_variant_get_data (v2), l1) == 0;
+}
+
 static OstreeStaticDeltaPartBuilder *
 allocate_part (OstreeStaticDeltaBuilder *builder)
 {
@@ -68,10 +151,57 @@ allocate_part (OstreeStaticDeltaBuilder *builder)
   part->payload = g_string_new (NULL);
   part->operations = g_string_new (NULL);
   part->uncompressed_size = 0;
+  part->mode_set = g_hash_table_new_full (mode_chunk_hash, mode_chunk_equals,
+                                          (GDestroyNotify)g_variant_unref, NULL);
+  part->modes = g_ptr_array_new ();
+  part->xattr_set = g_hash_table_new_full (xattr_chunk_hash, xattr_chunk_equals,
+                                           (GDestroyNotify)g_variant_unref, NULL);
+  part->xattrs = g_ptr_array_new ();
   g_ptr_array_add (builder->parts, part);
   return part;
 }
 
+static gsize
+allocate_part_buffer_space (OstreeStaticDeltaPartBuilder  *current_part,
+                            guint                          len)
+{
+  gsize empty_space;
+  gsize old_len;
+
+  old_len = current_part->payload->len;
+  empty_space = current_part->payload->allocated_len - current_part->payload->len;
+
+  if (empty_space < len)
+    {
+      gsize origlen;
+      origlen = current_part->payload->len;
+      g_string_set_size (current_part->payload, current_part->payload->allocated_len + (len - empty_space));
+      current_part->payload->len = origlen;
+    }
+
+  return old_len;
+}
+
+static gsize
+write_unique_variant_chunk (OstreeStaticDeltaPartBuilder *current_part,
+                            GHashTable                   *hash,
+                            GPtrArray                    *ordered,
+                            GVariant                     *key)
+{
+  gpointer target_offsetp;
+  gsize offset;
+
+  if (g_hash_table_lookup_extended (hash, key, NULL, &target_offsetp))
+    return GPOINTER_TO_UINT (target_offsetp);
+
+  offset = ordered->len;
+  target_offsetp = GUINT_TO_POINTER (offset);
+  g_hash_table_insert (hash, g_variant_ref (key), target_offsetp);
+  g_ptr_array_add (ordered, key);
+
+  return offset;
+}
+
 static GBytes *
 objtype_checksum_array_new (GPtrArray *objects)
 {
@@ -97,6 +227,37 @@ objtype_checksum_array_new (GPtrArray *objects)
   return g_byte_array_free_to_bytes (ret);
 }
 
+static gboolean
+splice_stream_to_payload (OstreeStaticDeltaPartBuilder  *current_part,
+                          GInputStream                  *istream,
+                          GCancellable                  *cancellable,
+                          GError                       **error)
+{
+  gboolean ret = FALSE;
+  const guint readlen = 4096;
+  gsize bytes_read;
+
+  while (TRUE)
+    {
+      allocate_part_buffer_space (current_part, readlen);
+
+      if (!g_input_stream_read_all (istream,
+                                    current_part->payload->str + current_part->payload->len,
+                                    readlen,
+                                    &bytes_read,
+                                    cancellable, error))
+        goto out;
+      if (bytes_read == 0)
+        break;
+          
+      current_part->payload->len += bytes_read;
+    }
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
 static gboolean
 process_one_object (OstreeRepo                       *repo,
                     OstreeStaticDeltaBuilder         *builder,
@@ -108,17 +269,27 @@ process_one_object (OstreeRepo                       *repo,
 {
   gboolean ret = FALSE;
   guint64 content_size;
-  gsize object_payload_start;
   gs_unref_object GInputStream *content_stream = NULL;
-  gsize bytes_read;
-  const guint readlen = 4096;
+  gs_unref_object GFileInfo *content_finfo = NULL;
+  gs_unref_variant GVariant *content_xattrs = NULL;
   guint64 compressed_size;
   OstreeStaticDeltaPartBuilder *current_part = *current_part_val;
 
-  if (!ostree_repo_load_object_stream (repo, objtype, checksum,
-                                       &content_stream, &content_size,
-                                       cancellable, error))
-    goto out;
+  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+    {
+      if (!ostree_repo_load_object_stream (repo, objtype, checksum,
+                                           &content_stream, &content_size,
+                                           cancellable, error))
+        goto out;
+    }
+  else
+    {
+      if (!ostree_repo_load_file (repo, checksum, &content_stream,
+                                  &content_finfo, &content_xattrs,
+                                  cancellable, error))
+        goto out;
+      content_size = g_file_info_get_size (content_finfo);
+    }
   
   /* Check to see if this delta is maximum size */
   if (current_part->objects->len > 0 &&
@@ -137,42 +308,71 @@ process_one_object (OstreeRepo                       *repo,
 
   g_ptr_array_add (current_part->objects, ostree_object_name_serialize (checksum, objtype));
 
-  object_payload_start = current_part->payload->len;
+  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+    {
+      gsize object_payload_start;
 
-  while (TRUE)
+      object_payload_start = current_part->payload->len;
+
+      if (!splice_stream_to_payload (current_part, content_stream,
+                                     cancellable, error))
+        goto out;
+
+      g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE);
+      _ostree_write_varuint64 (current_part->operations, content_size);
+      _ostree_write_varuint64 (current_part->operations, object_payload_start);
+    }
+  else
     {
-      gsize empty_space;
+      gsize mode_offset, xattr_offset, content_offset;
+      guint32 uid =
+        g_file_info_get_attribute_uint32 (content_finfo, "unix::uid");
+      guint32 gid =
+        g_file_info_get_attribute_uint32 (content_finfo, "unix::gid");
+      guint32 mode =
+        g_file_info_get_attribute_uint32 (content_finfo, "unix::mode");
+      gs_unref_variant GVariant *modev
+        = g_variant_ref_sink (g_variant_new ("(uuu)", 
+                                             GUINT32_TO_BE (uid),
+                                             GUINT32_TO_BE (gid),
+                                             GUINT32_TO_BE (mode)));
+
+      mode_offset = write_unique_variant_chunk (current_part,
+                                                current_part->mode_set,
+                                                current_part->modes,
+                                                modev);
+      xattr_offset = write_unique_variant_chunk (current_part,
+                                                 current_part->xattr_set,
+                                                 current_part->xattrs,
+                                                 content_xattrs);
+
+      if (S_ISLNK (mode))
+        {
+          const char *target;
 
-      empty_space = current_part->payload->allocated_len - current_part->payload->len;
-      if (empty_space < readlen)
+          g_assert (content_stream == NULL);
+
+          target = g_file_info_get_symlink_target (content_finfo);
+          content_stream = 
+            g_memory_input_stream_new_from_data (target, strlen (target), NULL);
+          content_size = strlen (target);
+        }
+      else
         {
-          gsize origlen;
-          origlen = current_part->payload->len;
-          g_string_set_size (current_part->payload, current_part->payload->allocated_len + (readlen - empty_space));
-          current_part->payload->len = origlen;
+          g_assert (S_ISREG (mode));
         }
 
-      if (!g_input_stream_read_all (content_stream,
-                                    current_part->payload->str + current_part->payload->len,
-                                    readlen,
-                                    &bytes_read,
-                                    cancellable, error))
+      content_offset = current_part->payload->len;
+      if (!splice_stream_to_payload (current_part, content_stream,
+                                     cancellable, error))
         goto out;
-      if (bytes_read == 0)
-        break;
-          
-      current_part->payload->len += bytes_read;
+
+      g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE);
+      _ostree_write_varuint64 (current_part->operations, mode_offset);
+      _ostree_write_varuint64 (current_part->operations, xattr_offset);
+      _ostree_write_varuint64 (current_part->operations, content_size);
+      _ostree_write_varuint64 (current_part->operations, content_offset);
     }
-      
-  /* A little lame here to duplicate the content size - but if in the
-   * future we do rsync-style rolling checksums, then we'll have
-   * multiple write calls.
-   */
-  _ostree_write_varuint64 (current_part->operations, content_size);
-  g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_WRITE);
-  _ostree_write_varuint64 (current_part->operations, object_payload_start);
-  _ostree_write_varuint64 (current_part->operations, content_size);
-  g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_CLOSE);
 
   ret = TRUE;
  out:
@@ -506,15 +706,26 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       gs_unref_variant GVariant *delta_part_content = NULL;
       gs_unref_variant GVariant *delta_part = NULL;
       gs_unref_variant GVariant *delta_part_header = NULL;
+      GVariantBuilder *mode_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(uuu)"));
+      GVariantBuilder *xattr_builder = g_variant_builder_new (G_VARIANT_TYPE ("aa(ayay)"));
       guint8 compression_type_char;
 
+      { guint j;
+        for (j = 0; j < part_builder->modes->len; j++)
+          g_variant_builder_add_value (mode_builder, part_builder->modes->pdata[j]);
+        
+        for (j = 0; j < part_builder->xattrs->len; j++)
+          g_variant_builder_add_value (xattr_builder, part_builder->xattrs->pdata[j]);
+      }
+        
       payload_b = g_string_free_to_bytes (part_builder->payload);
       part_builder->payload = NULL;
       
       operations_b = g_string_free_to_bytes (part_builder->operations);
       part_builder->operations = NULL;
       /* FIXME - avoid duplicating memory here */
-      delta_part_content = g_variant_new ("(@ay@ay)",
+      delta_part_content = g_variant_new ("(a(uuu)aa(ayay)@ay@ay)",
+                                          mode_builder, xattr_builder,
                                           ot_gvariant_new_ay_bytes (payload_b),
                                           ot_gvariant_new_ay_bytes (operations_b));
       g_variant_ref_sink (delta_part_content);
index 4c0f8c140032b9e0a758ea7ada9f0e4ff78f8850..0661787f92ca261613427b6ac2b2c28dbc8ca6ac 100644 (file)
@@ -34,10 +34,12 @@ G_BEGIN_DECLS
  *
  *   y  compression type (0: none, 'x': lzma)
  *   ---
- *   ay data source
+ *   a(uuu) modes
+ *   aa(ayay) xattrs
+ *   ay raw data source
  *   ay operations
  */
-#define OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0 "(ayay)"
+#define OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0 "(a(uuu)aa(ayay)ayay)"
 
 /**
  * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
@@ -130,9 +132,10 @@ gboolean _ostree_static_delta_part_execute_finish (OstreeRepo      *repo,
                                                    GError         **error); 
 
 typedef enum {
-  OSTREE_STATIC_DELTA_OP_WRITE = 1,
-  OSTREE_STATIC_DELTA_OP_GUNZIP = 2,
-  OSTREE_STATIC_DELTA_OP_CLOSE = 3
+  OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE = 'S',
+  OSTREE_STATIC_DELTA_OP_OPEN = 'o',
+  OSTREE_STATIC_DELTA_OP_WRITE = 'w',
+  OSTREE_STATIC_DELTA_OP_CLOSE = 'c'
 } OstreeStaticDeltaOpCode;
 
 gboolean
index e7a445cf02646412494cac5ffd775e2c685cc114..4ca4002320e6c941ddc2b4c99a126fa86463b5e0 100644 (file)
@@ -27,6 +27,7 @@
 #include <gio/gunixoutputstream.h>
 #include <gio/gfiledescriptorbased.h>
 
+#include "ostree-core-private.h"
 #include "ostree-repo-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-lzma-decompressor.h"
@@ -44,6 +45,9 @@ typedef struct {
 
   const guint8   *opdata;
   guint           oplen;
+
+  GVariant       *mode_dict;
+  GVariant       *xattr_dict;
   
   gboolean        object_start;
   gboolean        caught_error;
@@ -51,8 +55,6 @@ typedef struct {
 
   OstreeObjectType output_objtype;
   const guint8   *output_target;
-  char           *output_tmp_path;
-  GOutputStream  *output_tmp_stream;
   const guint8   *input_target_csum;
 
   const guint8   *payload_data;
@@ -80,18 +82,9 @@ typedef struct  {
                                    GCancellable               *cancellable, \
                                    GError                    **error);
 
-OPPROTO(write)
-OPPROTO(gunzip)
-OPPROTO(close)
+OPPROTO(open_splice_and_close)
 #undef OPPROTO
 
-static OstreeStaticDeltaOperation op_dispatch_table[] = {
-  { "write", dispatch_write },
-  { "gunzip", dispatch_gunzip },
-  { "close", dispatch_close },
-  { NULL }
-};
-
 static gboolean
 read_varuint64 (StaticDeltaExecutionState  *state,
                 guint64                    *out_value,
@@ -117,13 +110,9 @@ open_output_target (StaticDeltaExecutionState   *state,
   gboolean ret = FALSE;
   guint8 *objcsum;
   char checksum[65];
-  guint64 object_size;
-  gs_unref_object GInputStream *content_in_stream = NULL;
 
   g_assert (state->checksums != NULL);
   g_assert (state->output_target == NULL);
-  g_assert (state->output_tmp_path == NULL);
-  g_assert (state->output_tmp_stream == NULL);
   g_assert (state->checksum_index < state->n_checksums);
 
   objcsum = (guint8*)state->checksums + (state->checksum_index * OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN);
@@ -136,15 +125,6 @@ open_output_target (StaticDeltaExecutionState   *state,
 
   ostree_checksum_inplace_from_bytes (state->output_target, checksum);
 
-  /* Object size is the first element of the opstream */
-  if (!read_varuint64 (state, &object_size, error))
-    goto out;
-
-  if (!gs_file_open_in_tmpdir_at (state->repo->tmp_dir_fd, 0644,
-                                  &state->output_tmp_path, &state->output_tmp_stream,
-                                  cancellable, error))
-    goto out;
-
   ret = TRUE;
  out:
   return ret;
@@ -195,6 +175,8 @@ _ostree_static_delta_part_execute_raw (OstreeRepo      *repo,
   gboolean ret = FALSE;
   guint8 *checksums_data;
   gs_unref_variant GVariant *checksums = NULL;
+  gs_unref_variant GVariant *mode_dict = NULL;
+  gs_unref_variant GVariant *xattr_dict = NULL;
   gs_unref_variant GVariant *payload = NULL;
   gs_unref_variant GVariant *ops = NULL;
   StaticDeltaExecutionState statedata = { 0, };
@@ -213,39 +195,39 @@ _ostree_static_delta_part_execute_raw (OstreeRepo      *repo,
   state->checksums = checksums_data;
   g_assert (state->n_checksums > 0);
 
-  g_variant_get (part, "(@ay@ay)", &payload, &ops);
+  g_variant_get (part, "(@a(uuu)@aa(ayay)@ay@ay)",
+                 &mode_dict,
+                 &xattr_dict,
+                 &payload, &ops);
+
+  state->mode_dict = mode_dict;
+  state->xattr_dict = xattr_dict;
 
   state->payload_data = g_variant_get_data (payload);
   state->payload_size = g_variant_get_size (payload);
 
   state->oplen = g_variant_n_children (ops);
   state->opdata = g_variant_get_data (ops);
-  state->object_start = TRUE;
+
   while (state->oplen > 0)
     {
       guint8 opcode;
-      OstreeStaticDeltaOperation *op;
-
-      if (state->object_start)
-        {
-          if (!open_output_target (state, cancellable, error))
-            goto out;
-          state->object_start = FALSE;
-        }
 
       opcode = state->opdata[0];
+      state->oplen--;
+      state->opdata++;
 
-      if (G_UNLIKELY (opcode == 0 || opcode >= G_N_ELEMENTS (op_dispatch_table)))
+      switch (opcode)
         {
+        case OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE:
+          if (!dispatch_open_splice_and_close (repo, state, cancellable, error))
+            goto out;
+          break;
+        default:
           g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
-                       "Out of range opcode %u at offset %u", opcode, n_executed);
+                       "Unknown opcode %u at offset %u", opcode, n_executed);
           goto out;
         }
-      op = &op_dispatch_table[opcode-1];
-      state->oplen--;
-      state->opdata++;
-      if (!op->func (repo, state, cancellable, error))
-        goto out;
 
       n_executed++;
     }
@@ -255,8 +237,6 @@ _ostree_static_delta_part_execute_raw (OstreeRepo      *repo,
 
   ret = TRUE;
  out:
-  g_clear_pointer (&state->output_tmp_path, g_free);
-  g_clear_object (&state->output_tmp_stream);
   return ret;
 }
 
@@ -439,174 +419,117 @@ validate_ofs (StaticDeltaExecutionState  *state,
 }
 
 static gboolean
-dispatch_write (OstreeRepo                 *repo,
-                StaticDeltaExecutionState  *state,
-                GCancellable               *cancellable,  
-                GError                    **error)
-{
-  gboolean ret = FALSE;
-  guint64 offset;
-  guint64 length;
-  gsize bytes_written;
-
-  if (G_UNLIKELY(state->oplen < 2))
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
-                   "Expected at least 2 bytes for write op");
-      goto out;
-    }
-  if (!read_varuint64 (state, &offset, error))
-    goto out;
-  if (!read_varuint64 (state, &length, error))
-    goto out;
-  
-  if (!validate_ofs (state, offset, length, error))
-    goto out;
-
-  if (!g_output_stream_write_all (state->output_tmp_stream,
-                                  state->payload_data + offset,
-                                  length,
-                                  &bytes_written,
-                                  cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  if (!ret)
-    g_prefix_error (error, "opcode write: ");
-  return ret;
-}
-
-static gboolean
-dispatch_gunzip (OstreeRepo                 *repo,
-                 StaticDeltaExecutionState  *state,
-                 GCancellable               *cancellable,  
-                 GError                    **error)
-{
-  gboolean ret = FALSE;
-  guint64 offset;
-  guint64 length;
-  gs_unref_object GConverter *zlib_decomp = NULL;
-  gs_unref_object GInputStream *payload_in = NULL;
-  gs_unref_object GInputStream *zlib_in = NULL;
-
-  if (G_UNLIKELY(state->oplen < 2))
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
-                   "Expected at least 2 bytes for gunzip op");
-      goto out;
-    }
-  if (!read_varuint64 (state, &offset, error))
-    goto out;
-  if (!read_varuint64 (state, &length, error))
-    goto out;
-
-  if (!validate_ofs (state, offset, length, error))
-    goto out;
-
-  payload_in = g_memory_input_stream_new_from_data (state->payload_data + offset, length, NULL);
-  zlib_decomp = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
-  zlib_in = g_converter_input_stream_new (payload_in, zlib_decomp);
-
-  if (0 > g_output_stream_splice (state->output_tmp_stream, zlib_in,
-                                  G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
-                                  cancellable, error))
-    goto out;
-
-  ret = TRUE;
- out:
-  if (!ret)
-    g_prefix_error (error, "opcode gunzip: ");
-  return ret;
-}
-
-static gboolean
-dispatch_close (OstreeRepo                 *repo,
-                StaticDeltaExecutionState  *state,
-                GCancellable               *cancellable,  
-                GError                    **error)
+dispatch_open_splice_and_close (OstreeRepo                 *repo,
+                                StaticDeltaExecutionState  *state,
+                                GCancellable               *cancellable,  
+                                GError                    **error)
 {
   gboolean ret = FALSE;
-  char tmp_checksum[65];
-
-  if (state->checksum_index == state->n_checksums)
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
-                   "Too many close operations");
-      goto out;
-    }
-
-  g_assert (state->output_tmp_stream);
+  char checksum[65];
 
-  if (!g_output_stream_close (state->output_tmp_stream, cancellable, error))
+  if (!open_output_target (state, cancellable, error))
     goto out;
 
-  g_clear_object (&state->output_tmp_stream);
-
-  ostree_checksum_inplace_from_bytes (state->output_target, tmp_checksum);
+  ostree_checksum_inplace_from_bytes (state->output_target, checksum);
 
   if (OSTREE_OBJECT_TYPE_IS_META (state->output_objtype))
     {
       gs_unref_variant GVariant *metadata = NULL;
-      gs_fd_close int fd = -1;
-      
-      g_assert (state->output_tmp_path);
+      guint64 offset;
+      guint64 length;
 
-      fd = openat (state->repo->tmp_dir_fd, state->output_tmp_path, O_RDONLY | O_CLOEXEC);
-      if (fd == -1)
-        {
-          gs_set_error_from_errno (error, errno);
-          goto out;
-        }
-
-      if (!ot_util_variant_map_fd (fd, 0,
-                                   ostree_metadata_variant_type (state->output_objtype),
-                                   TRUE, &metadata, error))
+      if (!read_varuint64 (state, &length, error))
         goto out;
-
-      /* Now get rid of the temporary */
-      (void) unlinkat (state->repo->tmp_dir_fd, state->output_tmp_path, 0);
-
-      if (!ostree_repo_write_metadata_trusted (repo, state->output_objtype, tmp_checksum,
-                                               metadata, cancellable, error))
+      if (!read_varuint64 (state, &offset, error))
+        goto out;
+      if (!validate_ofs (state, offset, length, error))
+        goto out;
+      
+      metadata = g_variant_new_from_data (ostree_metadata_variant_type (state->output_objtype),
+                                          state->payload_data + offset, length, TRUE, NULL, NULL);
+      
+      if (!ostree_repo_write_metadata_trusted (state->repo, state->output_objtype,
+                                               checksum,
+                                               metadata,
+                                               cancellable,
+                                               error))
         goto out;
     }
   else
     {
-      gs_unref_object GInputStream *instream = NULL;
-      int fd;
-      struct stat stbuf;
+      guint64 mode_offset;
+      guint64 xattr_offset;
+      guint64 content_size;
+      guint64 content_offset;
+      guint64 objlen;
+      gs_unref_object GInputStream *object_input = NULL;
+      gs_unref_object GInputStream *memin = NULL;
+      gs_unref_object GFileInfo *finfo = NULL;
+      gs_unref_variant GVariant *xattrs_buf = NULL;
+      GVariant *modev;
+      GVariant *xattrs;
+      guint32 uid, gid, mode;
+
+      if (!read_varuint64 (state, &mode_offset, error))
+        goto out;
+      if (!read_varuint64 (state, &xattr_offset, error))
+        goto out;
+      if (!read_varuint64 (state, &content_size, error))
+        goto out;
+      if (!read_varuint64 (state, &content_offset, error))
+        goto out;
 
-      if (!ot_openat_read_stream (state->repo->tmp_dir_fd,
-                                  state->output_tmp_path, FALSE,
-                                  &instream, cancellable, error))
+      if (!validate_ofs (state, content_offset, content_size, error))
         goto out;
 
-      fd = g_file_descriptor_based_get_fd (G_FILE_DESCRIPTOR_BASED (instream));
-      if (fstat (fd, &stbuf) == -1)
+      modev = g_variant_get_child_value (state->mode_dict, mode_offset);
+      g_variant_get (modev, "(uuu)", &uid, &gid, &mode);
+      uid = GUINT32_FROM_BE (uid);
+      gid = GUINT32_FROM_BE (gid);
+      mode = GUINT32_FROM_BE (mode);
+
+      xattrs = g_variant_get_child_value (state->xattr_dict, xattr_offset);
+
+      finfo = _ostree_header_gfile_info_new (mode, uid, gid);
+
+      if (S_ISLNK (mode))
         {
-          gs_set_error_from_errno (error, errno);
-          goto out;
+          gs_free char *nulterminated_target =
+            g_strndup ((char*)state->payload_data + content_offset, content_size);
+          g_file_info_set_symlink_target (finfo, nulterminated_target);
+        }
+      else
+        {
+          g_assert (S_ISREG (mode));
+          g_file_info_set_size (finfo, content_size);
+          memin = g_memory_input_stream_new_from_data (state->payload_data + content_offset, content_size, NULL);
         }
 
-      /* Now get rid of the temporary */
-      (void) unlinkat (state->repo->tmp_dir_fd, state->output_tmp_path, 0);
-
-      if (!ostree_repo_write_content_trusted (repo, tmp_checksum,
-                                              instream, stbuf.st_size,
+      /* FIXME - Lots of allocation and serialization/deserialization
+       * going on here.  We need an
+       * ostree_repo_write_content_trusted_raw() that takes raw
+       * parameters.
+       */
+      if (!ostree_raw_file_to_content_stream (memin, finfo, xattrs,
+                                              &object_input, &objlen,
                                               cancellable, error))
         goto out;
+
+      if (!ostree_repo_write_content_trusted (state->repo,
+                                              checksum,
+                                              object_input,
+                                              objlen,
+                                              cancellable,
+                                              error))
+        goto out;
     }
 
+  state->checksum_index++;
   state->output_target = NULL;
-  g_clear_pointer (&state->output_tmp_path, g_free);
 
-  state->object_start = TRUE;
-  state->checksum_index++;
-      
   ret = TRUE;
  out:
   if (!ret)
-    g_prefix_error (error, "opcode close: ");
+    g_prefix_error (error, "opcode open-splice-and-close: ");
   return ret;
 }